package com.xiaotu.spider.project.pageprocessor.nettv;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;

import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Component;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.model.NetTVModel;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.DateUtils;
import com.xiaotu.common.util.HttpUtils;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.common.util.StringUtil;
import com.xiaotu.spider.AUrlPageProcessor;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

/**
 * 搜狐最新三年网剧爬取：名称、集数、类型、导演、主演、语言、年份、简介、播放量、评分
 *
 * @auther 王艳龙
 * @create 2017/10/13
 */
@Component("SOHUNetTVPageProcessor")
public class SOHUNetTVPageProcessor extends AUrlPageProcessor
{
    //网剧列表的页面地址，"%s"为要替换的年份
    private static final String BASE_URL =
    		"http://so.tv.sohu.com/list_p1101_p20_p3_p4%s_p5_p6_p77_p80_p91_p10_p11_p12_p13.html";
    //数据范围，只需要爬取这二年的网剧，该变量将被逐个拼到url中
    private static final String[] YEARS = DateUtils.getRecentYears(3);

    /**
     * 页面内容被爬虫框架捕获后，将回调此方法，来完成页面数据的解析
     *
     * @param page
     */
    @Override
    public void process(Page page)
    {
        if (Constants.PageDataType.SEARCH_LIST.equals(this.getPageDataType(page)))//列表页面处理
        {
            this.listProcess(page);
            this.addNextPage(page);
        }
        else
            page.putField(NetTVModel.class.getName(), detailProcess(page));//详情页面处理
    }

    /**
     * 常规详情页面解析
     *
     * @param page
     */
    private NetTVModel detailProcess(Page page)
    {
        Html html = page.getHtml();
        Map<String, Object> tvMap = page.getRequest().getExtras();
        String playlistId = getSrings(html.toString(), Pattern.quote("var playlistId = \"") +
                "(.*?)" + Pattern.quote("\";"));
    	String jsonStr = HttpUtils.httpGetJSONP("http://pl.hd.sohu.com/videolist?playlistid="+playlistId+""
    			+ "&order=0&cnt=1&withLookPoint=1&preVideoRule=1&callback=__get_videolist");
    	jsonStr = jsonStr.substring(jsonStr.indexOf("{"));
		jsonStr = jsonStr.substring(0, jsonStr.lastIndexOf("}") + 1);
		JSONObject C_json = JSONObject.fromObject(jsonStr);
		
    	if(jsonStr.isEmpty())
    		throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
    		String title = C_json.getString("albumName");
    		String sets = C_json.getString("totalSet");
    		JSONArray directorsList = C_json.getJSONArray("directors");
    		String director = StringUtils.join(directorsList,",");
    		String area = C_json.getString("area");
    		JSONArray typeList = C_json.getJSONArray("categories");
    		String type = StringUtils.join(typeList,",");
    		String year = C_json.getString("publishYear");
    		String play = this.getPlayCount(playlistId);
    		String score = this.getScore(playlistId);
    		String desc = C_json.getString("albumDesc");
    		JSONArray actorsList = C_json.getJSONArray("actors");
    		String actor = StringUtils.join(actorsList,",");
    	return new NetTVModel(Constants.TargetType.OTHER_TYPE_ID, title, Constants.SiteName
    			.SOHU, sets, type, null, area, year, director, actor, desc, play, score,StringUtil.nullToString(tvMap.get("year")));
        	
    }

    /**
     * 将列表中的所有网剧添加到爬取池中
     *
     * @param page
     */
    private void listProcess(Page page)
    {
        List<String> list = page.getHtml().xpath("//div[@class='column-bd cfix']//ul//li").all();
        list.forEach(divStr -> PageUtils.addTargetRequest(page, new Html(divStr).xpath
                ("//div[@class='st-pic']/a/@href").get(), page.getRequest().getExtras()));
    }

    /**
     * 获取网剧播放量
     *
     * @param html
     * @return
     */
    private String getPlayCount(String playlistId)
    {
        String url ="http://count.vrs.sohu.com/count/query_Album.action?"
        		+ "albumId="+playlistId+""
        		+ "&jsonp=counttopInfo&type=2";
        String data = HttpUtils.httpGetJSONP(url);
        if (StringUtils.isEmpty(data))
            return StringUtils.EMPTY;
        String playCount = RegexUtils.regexSearch("counttopInfo=" + RegexUtils.REGEX_NUMBER, data);
        if (StringUtils.isEmpty(playCount))
            return StringUtils.EMPTY;
        return RegexUtils.regexSearch(RegexUtils.REGEX_NUMBER, playCount);
    }
    
    /**
     * 获取网剧评分
     *
     * @param html
     * @return
     */
    private String getScore(String playlistId)
    {
        String url ="http://vote.biz.itc.cn/query/score.json?callback=jQuery172010347057230044143_1507883239551"
        		+ "&albumId="+playlistId+""
        		+ "&_=1507883239750";
        System.out.println(url);
        String data = HttpUtils.httpGetJSONP(url);
        if (StringUtils.isEmpty(data))
            return StringUtils.EMPTY;
        String score = getSrings(data, Pattern.quote("\"score\":\"") +
                "(.*?)" + Pattern.quote("\","));
        if (StringUtils.isEmpty(score))
            return StringUtils.EMPTY;
        return RegexUtils.regexSearch(RegexUtils.REGEX_NUMBER, score);
    }

    /**
     * 将下一页网剧列表页面放入爬取池中
     *
     * @param page
     */
    private void addNextPage(Page page)
    {
        String href = page.getHtml().xpath("//div[@class='ssPages area']/a[@title='下一页']/@href")
                .get();
        if (StringUtils.isNotEmpty(href))
            PageUtils.addTargetRequest(page, href, page.getRequest().getExtras());

    }

    /**
     * 初始化要爬取的地址，本程序中为二个地址（对应二个年份）
     *
     * @return
     */
    @Override
    protected List<JSONObject> getStartRequestUrlsAndParams()
    {
        List<JSONObject> list = new ArrayList<>();
        Stream.of(YEARS).forEach(year ->
        {
            JSONObject target = new JSONObject();
            target.put("year", year);
            target.put(URL_KEY, String.format(BASE_URL, year));
            list.add(target);
        });
        return list;
    }
    
    /**
     * 获取多个目标字符串
     */
    public static String getSrings(String input, String regex)
    {
    	String result=null;
        Pattern pat = Pattern.compile(regex, Pattern.DOTALL);
        Matcher mat = pat.matcher(input);
        while (mat.find())
        {
            result = mat.group(1);
        }
        return result;
    }

    /**
     * 爬取链接为静态地址时，该方法返回空，固定写法
     *
     * @return
     */
    @Override
    protected String[] getStartRequestUrls()
    {
        return null;
    }

    /**
     * 爬取任务的标识
     *
     * @return
     */
    @Override
    protected String getSpiderType()
    {
        return "SOHU_NETTV";
    }

    /**
     * 返回本页面的数据类型
     *
     * @param page 爬取页面
     * @return
     */
    @Override
    public String getPageDataType(Page page)
    {
        if (page.getUrl().toString().contains("so.tv.sohu.com"))//网剧列表页面
            return Constants.PageDataType.SEARCH_LIST;
        return Constants.PageDataType.TV_RECORD;//网剧详情页面
    }
}
